﻿using System;
using System.Collections.Generic;
using System.Data;
using System.Linq;
using System.Text;
using System.Threading;
using WebCrawling.Data.Extracting.DB;

namespace WebCrawling.Data.Extracting
{
    public class Workflow
    {
        public static Dictionary<Uri, Rule> crawlUrls = new Dictionary<Uri, Rule>();
        public static Dictionary<Uri, Rule> linkUrls = new Dictionary<Uri, Rule>();
        public static Dictionary<Uri, Output> outputs = new Dictionary<Uri, Output>();
        public static List<Uri> scrapedUrls = new List<Uri>();
        public static MySqlDBUtil db;
        private static string sync = "";
        public static void Process(int MaxThreads)
        {
            InitDb();
            DataTable crawledData = db.getTableData(MySqlDBUtil.CRAWLURLTABLENAME).Tables[0];
            DataTable scrapedData = db.getTableData(MySqlDBUtil.SCRAPURLTABLENAME).Tables[0];
            ProcessCrawlUrls();
            ThreadPool threadPool = new ThreadPool(MaxThreads);

            //Crawing work
            threadPool.AddTask(new Task(Workflow.Crawling), crawledData);
            threadPool.WaitForTasksToComplete();

            //Scraping work
            if (MaxThreads > 1)
            {
                for (int i = 0; i < MaxThreads; i++)
                {
                    threadPool.AddTask(new Task(Workflow.Scraping), scrapedData);
                }
            }
            else
            {
                threadPool.AddTask(new Task(Workflow.Scraping), null);
            }

            threadPool.WaitForTasksToComplete();
        }

        public static void Crawling(object parameters)
        {
            DataTable crawledData = (DataTable)parameters;
            do
            {
                Uri url = null;
                try
                {
                    if (crawlUrls.Count > 0)
                    {
                        Monitor.Enter(sync);
                        KeyValuePair<Uri, Rule> firstRule = crawlUrls.First();
                        Rule rule = firstRule.Value;
                        url = firstRule.Key;
                        rule.CrawledURL = url.ToString();
                        crawlUrls.Remove(url);
                        Monitor.Pulse(sync);
                        Monitor.Exit(sync);
                        string html = "";
                        //if (!db.IsExist("select * from " + MySqlDBUtil.CRAWLURLTABLENAME + " where " + MySqlDBUtil.FIELDS[0] + "='" + url + "';"))
                        string find = "url = '" + url + "'";
                        DataRow[] foundRows = crawledData.Select(find);
                        if (foundRows.Count() == 1)
                        {
                            html = foundRows[0].ItemArray[2].ToString();
                        }
                        else
                        {
                            PageClass page = HttpRequestSyn.Instance.GetPage(url, rule.Charset);
                            html = page.html;
                            if (html.IsNotNullOrEmpty())
                            {
                                string[] fieldvalues = new string[MySqlDBUtil.FIELDS.Length];
                                fieldvalues[0] = url.ToString();
                                fieldvalues[1] = html;
                                db.insert(MySqlDBUtil.CRAWLURLTABLENAME, fieldvalues);
                            }
                        }

                        foreach (string link in html.GetMatchGroups(rule.LinksRegex))
                        {
                            string outputLink = link;
                            if (!outputLink.StartsWith(rule.URLRule.UrlPrefix))
                            {
                                if (outputLink.StartsWith("/"))
                                {
                                    outputLink = rule.URLRule.UrlPrefix + outputLink;
                                }
                                else
                                {
                                    outputLink = rule.URLRule.UrlPrefix + "/" + outputLink;
                                }

                            }
                            Utility.Log(outputLink, String.Format("Get this request URL from {0}", url));
                            if (!linkUrls.Keys.Contains(new Uri(outputLink)))
                            {
                                linkUrls.Add(new Uri(outputLink), rule);
                                Utility.SiteSpecialProcessing(html, new Uri(outputLink), outputs, SiteType.FatSecret);
                            }
                        }
                    }
                    else
                    {
                        break;
                    }
                }
                catch (Exception ex)
                {
                    Utility.Log((url == null ? "" : url.ToString()), ex.Message);
                }
            } while (true);
        }

        public static void Scraping(object parameters)
        {
            int times = 0;
            DataTable scrapedData = (DataTable)parameters;
            do
            {
                Uri url = null;
                try
                {

                    if (linkUrls.Count > 0)
                    {
                        Monitor.Enter(sync);
                        KeyValuePair<Uri, Rule> firstRule = linkUrls.First();
                        url = firstRule.Key;
                        Rule rule = firstRule.Value;
                        rule.LinkedURL = url.ToString();
                        linkUrls.Remove(url);
                        Monitor.Pulse(sync);
                        Monitor.Exit(sync);
                        times = 0;
                        string html = "";
                        string find = "url = '" + url + "'";
                        DataRow[] foundRows = scrapedData.Select(find);
                        if (foundRows.Count() == 1)
                        {
                            html = foundRows[0].ItemArray[2].ToString();
                        }
                        else
                        {
                            PageClass page = HttpRequestSyn.Instance.GetPage(url, rule.Charset);
                            html = page.html;
                            string[] fieldvalues = new string[MySqlDBUtil.FIELDS.Length];
                            fieldvalues[0] = url.ToString();
                            fieldvalues[1] = html;
                            db.insert(MySqlDBUtil.SCRAPURLTABLENAME, fieldvalues);
                        }
                        Output output;
                        bool exist = false;
                        if (outputs.ContainsKey(url))
                        {
                            exist = true;
                            output = outputs[url];
                        }
                        else
                        {
                            output = new Output();
                            output.Url = url;
                        }
                        Dictionary<string, string> temp = new Dictionary<string, string>();
                        foreach (var t in output.Fields)
                        {
                            temp.Add(t.Key, t.Value);
                        }
                        output.Fields.Clear();

                        foreach (var fieldAttributes in rule.Fields)
                        {
                            if (temp.ContainsKey(fieldAttributes.Key))
                            {
                                output.Fields.Add(fieldAttributes.Key, temp[fieldAttributes.Key]);
                            }
                            else
                            {
                                output.Fields.Add(fieldAttributes.Key, html.GetMatchValue(fieldAttributes.Value.Regex, fieldAttributes.Value.RemoveHtmlTags).ContentSpecialProcessing(firstRule));
                            }
                        }

                        if (exist)
                        {
                            outputs[url] = output;
                        }
                        else
                        {
                            outputs.Add(url, output);
                        }
                        Monitor.Enter(sync);
                        Outputing(output, rule);
                        Monitor.Pulse(sync);
                        Monitor.Exit(sync);
                    }
                    else
                    {
                        Thread.Sleep(5000);
                        times++;
                    }
                }
                catch (Exception ex)
                {
                    Utility.Log((url == null ? "" : url.ToString()), ex.Message);
                }
                if (times > 2)
                {
                    break;
                }

            } while (true);
        }

        public static void Outputing(Output output, Rule rule)
        {
            Dictionary<string, string> listOutputField = new Dictionary<string, string>();
            Utility.AppendTextToFile(Utility.SCRAPEDURLSPATH, output.Url.ToString());
            switch (rule.URLRule.SaveFormatType)
            {
                case SaveFormatType.DiscuzArticle:
                    string title = "";
                    string content = "";
                    foreach (KeyValuePair<string, string> field in output.Fields)
                    {
                        if (field.Key == "Title")
                        {
                            title = "'" + field.Value.Replace("'", "''") + "'";
                        }
                        if (field.Key == "Content")
                        {
                            content = "'" + field.Value.Replace("'", "''") + "'";
                        }
                    }
                    if (title.IsNotNullOrEmpty() && content.IsNotNullOrEmpty())
                    {
                        string summary = content.RemoveHtmlTags();
                        if (summary.Length > 100)
                        {
                            summary = "'" + summary.Substring(0, 100) + "'";
                        }
                        listOutputField.Add("title", title);
                        listOutputField.Add("dateline", "UNIX_TIMESTAMP(NOW())");
                        listOutputField.Add("allowcomment", "1");
                        listOutputField.Add("summary", summary);
                        listOutputField.Add("catid", rule.URLRule.Category);
                        listOutputField.Add("uid", "1");
                        listOutputField.Add("id", "0");
                        listOutputField.Add("", "");

                        Utility.WriteSql("article.sql", output, rule.URLRule.Category);
                    }
                    else
                    {
                        Utility.Log(rule.Name, "Can't parse this page");
                    }
                    break;
                case SaveFormatType.DiscuzForum:
                    Utility.WriteSql("forum.sql", output, rule.URLRule.Category);
                    break;
                case SaveFormatType.CSV:
                    List<string> list = new List<string>();
                    list.Add(rule.CrawledURL);
                    list.Add(rule.LinkedURL);
                    foreach (KeyValuePair<string, string> field in output.Fields)
                    {
                        list.Add(field.Value);
                    }
                    list.SaveAsCsv("foods.csv");
                    break;
            }

        }

        //public static void SaveOutputs()
        //{
        //    List<Outputs> list = new List<Outputs>();
        //    foreach (Outputs output in Outputs.Instance.DicOutputs().Values)
        //    {
        //        list.Add(output);
        //    }
        //    list.ToDataTable().SaveAsCsv("data.csv");
        //}
        public static void getScrapedUrls()
        {

        }

        public static void InitDb()
        {
            MySqlConfig config = new MySqlConfig();
            config.ServerName = "127.0.0.1";
            config.UserID = "root";
            config.Password = "";
            config.DataBaseName = MySqlDBUtil.DTATBASENAME;

            db = new MySqlDBUtil(config, MySqlDBUtil.SCRAPURLTABLENAME, MySqlDBUtil.FIELDS);
        }

        public static void ProcessCrawlUrls()
        {
            foreach (Rule rule in Rules.Instance.DicRules.Values)
            {
                if (rule.Status == Status.Queueing)
                {
                    List<string> UrlFormats = rule.URLRule.UrlFormat.Split('|').ToList();
                    string crawUrl = "";
                    foreach (string UrlFormat in UrlFormats)
                    {
                        crawUrl = UrlFormat.Replace("{UrlPrefix}", rule.URLRule.UrlPrefix);
                        string maxPage = crawUrl.GetMatchValue("{MaxPage=(.+?)}");
                        if (maxPage.IsNotNullOrEmpty())
                        {
                            for (int i = Int16.Parse(maxPage); i > 1; i--)
                            {
                                crawlUrls.Add(new Uri(crawUrl.Replace("{MaxPage=" + maxPage + "}", i.ToString())), rule);
                            }
                        }
                        else
                        {
                            crawlUrls.Add(new Uri(crawUrl), rule);
                        }

                    }
                    //crawlUrls.Add(new Uri("http://www.fatsecret.com/Default.aspx?pa=toc&pg=302&f=a&s=0"), rule);
                    //crawlUrls.Add(new Uri("http://www.fatsecret.com/Default.aspx?pa=toc&s=2"), rule);
                }
            }
        }

        public class GoogleCrawing
        {
            public static List<string> Crawing(Rule rule)
            {
                return GoogleService.GetSearchUrls(rule);
            }
        }
    }
}
